{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# And for visualization on Colab install\n",
        "# !apt-get install x11-utils > /dev/null 2>&1 \n",
        "# !pip install pyglet\n",
        "# !apt-get install -y xvfb python-opengl > /dev/null 2>&1\n",
        "# !pip install gym pyvirtualdisplay > /dev/null 2>&1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tOhAux1ubEKG",
        "outputId": "727c632b-d755-4ac2-dd1a-9b84c4f1259f"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyglet in /usr/local/lib/python3.7/dist-packages (1.5.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet) (0.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "JZV-qP-yay8_"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import gym\n",
        "#import math\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense\n",
        "from tensorflow.keras.optimizers import Adam\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Uncomment if working on Colab\n",
        "# from pyvirtualdisplay import Display\n",
        "# display = Display(visible=0, size=(600, 400))\n",
        "# display.start()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i63z1vW0c4Sp",
        "outputId": "222984a5-6556-4942-a509-c5a1a639c63f"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyvirtualdisplay.display.Display at 0x7f0d25df98d0>"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "ikpmIrLyay9B"
      },
      "outputs": [],
      "source": [
        "EPOCHS = 1000\n",
        "THRESHOLD = 45\n",
        "MONITOR = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "trKmD7d2ay9C"
      },
      "outputs": [],
      "source": [
        "class DQN():\n",
        "    def __init__(self, env_string,batch_size=64):\n",
        "        self.memory = deque(maxlen=100000)\n",
        "        self.env = gym.make(env_string)\n",
        "        input_size = self.env.observation_space.shape[0]\n",
        "        action_size = self.env.action_space.n\n",
        "        self.batch_size = batch_size\n",
        "        self.gamma = 1.0\n",
        "        self.epsilon = 1.0\n",
        "        self.epsilon_min = 0.01\n",
        "        self.epsilon_decay = 0.995\n",
        "        \n",
        "        alpha=0.01\n",
        "        alpha_decay=0.01\n",
        "        if MONITOR: self.env = gym.wrappers.Monitor(self.env, 'data/'+env_string, force=True)\n",
        "        \n",
        "        # Init model\n",
        "        self.model = Sequential()\n",
        "        self.model.add(Dense(24, input_dim=input_size, activation='tanh'))\n",
        "        self.model.add(Dense(48, activation='tanh'))\n",
        "        self.model.add(Dense(action_size, activation='linear'))\n",
        "        self.model.compile(loss='mse', optimizer=Adam(lr=alpha, decay=alpha_decay))\n",
        "\n",
        "    def remember(self, state, action, reward, next_state, done):\n",
        "        self.memory.append((state, action, reward, next_state, done))\n",
        "\n",
        "    def choose_action(self, state, epsilon):\n",
        "        if np.random.random() <= epsilon:\n",
        "            return self.env.action_space.sample()\n",
        "        else:\n",
        "            return np.argmax(self.model.predict(state))\n",
        "\n",
        "    def preprocess_state(self, state):\n",
        "        return np.reshape(state, [1, 4])\n",
        "\n",
        "    def replay(self, batch_size):\n",
        "        x_batch, y_batch = [], []\n",
        "        minibatch = random.sample(self.memory, min(len(self.memory), batch_size))\n",
        "        for state, action, reward, next_state, done in minibatch:\n",
        "            y_target = self.model.predict(state)\n",
        "            y_target[0][action] = reward if done else reward + self.gamma * np.max(self.model.predict(next_state)[0])\n",
        "            x_batch.append(state[0])\n",
        "            y_batch.append(y_target[0])\n",
        "        \n",
        "        self.model.fit(np.array(x_batch), np.array(y_batch), batch_size=len(x_batch), verbose=0)\n",
        "        #epsilon = max(epsilon_min, epsilon_decay*epsilon) # decrease epsilon\n",
        "       \n",
        "\n",
        "    def train(self):\n",
        "        scores = deque(maxlen=100)\n",
        "        avg_scores = []\n",
        "        \n",
        "\n",
        "        for e in range(EPOCHS):\n",
        "            state = self.env.reset()\n",
        "            state = self.preprocess_state(state)\n",
        "            done = False\n",
        "            i = 0\n",
        "            while not done:\n",
        "                action = self.choose_action(state,self.epsilon)\n",
        "                next_state, reward, done, _ = self.env.step(action)\n",
        "                next_state = self.preprocess_state(next_state)\n",
        "                self.remember(state, action, reward, next_state, done)\n",
        "                state = next_state\n",
        "                self.epsilon = max(self.epsilon_min, self.epsilon_decay*self.epsilon) # decrease epsilon\n",
        "                i += 1\n",
        "\n",
        "            scores.append(i)\n",
        "            mean_score = np.mean(scores)\n",
        "            avg_scores.append(mean_score)\n",
        "            if mean_score >= THRESHOLD and e >= 100:\n",
        "                print('Ran {} episodes. Solved after {} trials ✔'.format(e, e - 100))\n",
        "                return avg_scores\n",
        "            if e % 100 == 0:\n",
        "                print('[Episode {}] - Mean survival time over last 100 episodes was {} ticks.'.format(e, mean_score))\n",
        "\n",
        "            self.replay(self.batch_size)\n",
        "        \n",
        "        print('Did not solve after {} episodes 😞'.format(e))\n",
        "        return avg_scores\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4STstW_7ay9E",
        "outputId": "b9a26bf3-dd8c-4b4f-c92c-9b6a75333acf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Episode 0] - Mean survival time over last 100 episodes was 28.0 ticks.\n",
            "[Episode 100] - Mean survival time over last 100 episodes was 15.71 ticks.\n",
            "[Episode 200] - Mean survival time over last 100 episodes was 27.81 ticks.\n",
            "Ran 259 episodes. Solved after 159 trials ✔\n"
          ]
        }
      ],
      "source": [
        "env_string = 'CartPole-v0'\n",
        "agent = DQN(env_string)\n",
        "scores = agent.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "28iEbGwzay9F",
        "outputId": "e9ab9177-f5eb-472f-dead-64a17be16cf7"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9bn48c+TyUpWsoeELew7hLCIYHGrFje07tWiVemmV9veapd7W9t7e9vaa6WLt1argNaKrejPfa0oBRQI+xoIIYSE7JCdTDIz398fM8EgCQnJLDmT5/165eXMmZlznuMJT77znO8ixhiUUkpZT0igA1BKKdU7msCVUsqiNIErpZRFaQJXSimL0gSulFIWFerPgyUnJ5sRI0b485BKKWV5W7ZsqTbGpHx+u18T+IgRI8jLy/PnIZVSyvJE5Ehn27WEopRSFqUJXCmlLEoTuFJKWZQmcKWUsihN4EopZVGawJVSyqI0gSullEVpAldKKR+qrG/hf9/Np7Cq0ev71gSulFI+VFDVyB/XFFBe1+L1fWsCV0opH2pP3BkJUV7ftyZwpZTyoTJPAk+Pi/T6vnucwEXEJiLbROQNz/MVInJYRLZ7fqZ7PTqllLK4srqTJAwKIyrc5vV9n8tkVvcD+4C4Dtu+b4x5ybshKaVU8Civa/FJ6xt62AIXkSzgCuAvPolCKaWCVFldCxnxAUzgwDLgQcD1ue2/EJGdIvKYiER09kERWSoieSKSV1VV1ZdYlVLKcsrrWnxyAxN6kMBF5Eqg0hiz5XMv/RAYD8wCEoGHOvu8MeZJY0yuMSY3JeWM+ciVUipotbQ5qWlqJSOAJZTzgatFpAhYBVwkIn81xpQZNzuwHJjtkwiVUsqiKuo9PVACVUIxxvzQGJNljBkB3Ax8aIy5TUQyAEREgMXAbp9EqJRSFtXehTAj3jcllL4sqfa8iKQAAmwHvuGdkJRSKji0D+LxVQv8nBK4MeYj4CPP44t8EI9SSgWNz1rgge2FopRS6hyV150kLjKU6AjfrB+vCVwppXzE3QfcN/Vv0ASulFI+U1bX4rP6N2gCV0opnymra2FIgiZwpZSylFaHi+pGO+lxWkJRSilLaR/E46seKKAJXCmlfKLcx6MwQRO4Ukr5xLHak4C2wJVSynJ8uZRaO03gSinlA2V1LcRGhBLjo0E8oAlcKaV8otzHfcBBE7hSSvlEWb0mcKWUsqTyupMM8eEwetAErpRSXtfmdFHZYNcWuFJKWU1lgx1jfNuFEDSBK6WU15XXufuAawtcKaUsxtdLqbXTBK6UUl722SAebYErpZSlHKttITrcRqwPB/GAJnCllPK68vqTpMdHIiI+PU6PE7iI2ERkm4i84Xk+UkQ2ikiBiLwoIuG+C1MppazjcHUzWYMH+fw459ICvx/Y1+H5r4HHjDGjgRPAXd4MTCmlrKilzcnBigYmZ8b5/Fg9SuAikgVcAfzF81yAi4CXPG9ZCSz2RYBKKWUl+8rqcbgMUzITfH6snrbAlwEPAi7P8ySg1hjj8DwvATI7+6CILBWRPBHJq6qq6lOwSinV3+0qrQNgala8z4/VbQIXkSuBSmPMlt4cwBjzpDEm1xiTm5KS0ptdKKWUZewqqSMpOtznozABetLH5XzgahFZBEQCccDvgAQRCfW0wrOAUt+FqZRS1rCrtI4pWfE+74ECPWiBG2N+aIzJMsaMAG4GPjTGfAVYA1zvedsS4FWfRamUUhbgchkKq5oYmxbrl+P1pR/4Q8B3RaQAd038ae+EpJRS1lTT1Eqr08UQP5RPoGcllFOMMR8BH3keFwKzvR+SUkpZU5lnEqshPlwHsyMdiamUUl7SvhK9JnCllLKYY7XtsxD6p4SiCVwppbykrO4kEaEhJEb7Z2YRTeBKKeUlx+paGJIQ5ZcuhKAJXCmlvOZY7Um/lU9AE7hSSnlNWW2Lz1fh6UgTuFJKeYF7JfoWhvh4FZ6ONIErpZQXHKhowGVgWKLv5wFvpwlcKaW84O1d5YQIXDg+1W/H1ASulFJ9ZIzhrV1lnDcqieSYCL8dN2gTeMmJZvYeqw90GEqpAWB/eQOF1U0smpLh1+MGbQL/4cu7WPpcXqDDUEoNAG/uLCNE4LJJ6X49rm/XvA+Qk61ONh4+TqvDRUNLG7GRYYEOSSkVpAJVPoEgbYF/eriGVod79beDlY0BjkYpFcwCVT6BIE3gaw9U0T6S9WBFQ2CDUUoFtRc2FRMaIn4vn0CQJvDNRceZOzKJiNAQ8su1Ba6U8o3S2pOs2nSUG3KH+r18AkGawBtaHKTFRTAmLYaDldoCV0r5xu8+OADAfReNDsjxgzKB29tcRITaGJsaywEtoSilfGB3aR3/2FLCV88b7rcFHD4vOBO4w0lEWAhj0mKpqLdTd7It0CEppSzC6TLklzfgcLq6fI/LZfjZ63sYPCic+y4e48foTheU3QjtDhcRoSGMTYsB3Dcyc0ckBjgqpVR/t2Z/JT97fQ9FNc2MTo1h8fQh3JA7lLS40yeoWvlJEZuLTvDI9VOJjwpcN+VuW+AiEikim0Rkh4jsEZGfebavEJHDIrLd8zPd9+H2jDuB2xibFgvAgQq9kamUOrtlHxzgzhWbCQ8N4ceLJhBmC+F/3zvAN/+65bT3FVU38et39rNwXAo3zMwKULRuPWmB24GLjDGNIhIGrBORtz2vfd8Y85Lvwjt3DqcLp8sQERpCZkIUg8JtWgdXSp3VM+sOs+yDg3w5J4tfXDuZyDAb91yQzTPrDvPzN/ayq6SOKVnxOF2GB1fvJMwWwi+vm+K3lXe60m0L3Li1N2HDPD/Gp1H1gd0zgCciLISQEGFMaowmcKVUlyrqW3jk3f1cPD6VR66fSmSY7dRr1+dmMSjcxvL1h3G5DA++tJNNh4/zkysn+nXhhq706CamiNhEZDtQCbxvjNnoeekXIrJTRB4TEf93guzEqQQe6r4IY9JitYSilOrSsg8O4HQZfnrVJGwhp7eo4yLDuG3ucF7eVsrtz2xk9dYSvnPJWG7IHRqgaE/XowRujHEaY6YDWcBsEZkM/BAYD8wCEoGHOvusiCwVkTwRyauqqvJS2F2zO5wARIS6T21sWgzVjXZONLX6/NhKKWspqGzgxc1H+cqc4QxL6nwhhu9eOpbx6bGsL6jh2xeO4t8uDkyf786cUzdCY0wtsAa43BhT5imv2IHlwOwuPvOkMSbXGJObkpLS94i7YW/7rIQCMCbVfSOzoEpb4Uqp0/36nXyiw0PPOhAnMszGijtn839fyeHfvzgu4HXvjnrSCyVFRBI8j6OAS4H9IpLh2SbAYmC3LwPtqc+XUEYmRwNwuLopYDEppfqfzUXHeX9vBd9YOIqkbobBp8dHsmhKRr9K3tCzXigZwEoRseFO+H83xrwhIh+KSAogwHbgGz6Ms8da2k4voWQNjiI0RCjSBK6U8jDG8D9v7SMtLoKvnT8y0OH0WrcJ3BizE5jRyfaLfBJRH32+BR5qC2Fo4iCKajSBK6Xc3t1TzrbiWn513RSiwm3df6CfCrqh9KduYoZ9dmojkgZxuLo5UCEppfqRNqeLR97JZ3RqDNcHeCBOXwVfAm+/iRnaIYEnR3Okpglj+m33daVUDzhdhmfWHWZDQXWv9/G3jcUUVjfx0OXjCbVZOwUG3Vwony+hgPtGZnOrk8oG+xlzGiilrKHR7uCBVdv5YF8FUWE2Xv7WPCZkxJ3TPqoa7Pzve/nMG5XEJRNSfRSp/1j7z08nPt8PHGBEkvZEUcrKSk40c/2fNvDh/gq+d+lYYiNDWfT7f3H3ys04XT3/Zv2njw7R0ubk59dM7nc9SnojCBO4uwXecThse1fCwipN4EpZSVWDnZUbilj8+HpKa0+y/M7Z3HfxGF76xjxumzOcD/ZV8lF+ZY/391F+JeePTmZ0aowPo/af4CuhtJ3ZAs9MiCJaJ7VSyjL2Hqtn2QcH+GBfBS4DkzPjWHbTdEZ7BuYNSxrET66ayHt7y1mxoYiLJ6R1u8+yupMUVjdx65xhvg7fb4IvgTtOH4kJuCe1Sotlf3l9oMJSSvXQ6zuO8b2/7yAyLISvf2EUi6dnMi499oz3hdlC+Mqc4fz2/QMcqmpkVMrZW9XrC2oAmDcq2SdxB0LQllDCP3d3eXx6LPnlDdoTRal+6kBFA/ev2sZ9L2xj+tAE1j54IQ9dPr7T5N3ultnDCLeF8NwnR7rd/4aCahKjwxl/lv1ZTRC2wJ2EhsgZ3YPGpsWyavNRqhrtpMZqTxSl+gOH08XfNhXzt43F7C9vIDw0hPsuGs23Lxx92n2srqTERnDF1Axe2lLCv182jpiIzlNak93B+3sruHRSGiEh1r952S74Enib67T6d7v2v7r55Q2awJXqB5rsDu5YvonNRSeYMSyB/7xyItfOyCQxOvyc9nPb3OG8sq2U9/aUc11O5wNzXt9xjAa7g1tnB0/9G4IxgTtcRHTyl3tchwS+YIzvZ0VUSnXN6TIsfS6PrcW1PHrDNK7Lyex1t74ZQxOIjwrj08KaThN4o93B8vVFjE+PZebwwX0NvV8Jwhq4s9MWeFJMBKmxEewurQtAVEqpjp5Zd5j1BTX8YvFkvjwzq099skNChNkjE9l4+PgZr7W0ObnxiU8oqGrkgUvGBkXf746CMIF3XkIByBk2mK3FtX6OSCnVzhjD6i0l/Oa9fC6dmMZNs7yzss2ckYkcqWmmvK7ltO3v7a1gb1k9y26azuWT071yrP7E8gn8o/xKvvPidhxOd+8Tdw2885sfOcMTKD7eTFWD3Z8hKqU8/vKvw3zvHzuYlhXv1UWB52YnAfCDl3eyvsM8Ka9uKyUjPpIrpmR45Tj9jWUS+PGmVlZtKj5j+x8+LOCVbaU8v9H9mt3hPK0PeEc5w9z1r63FJ3wXqFKqU+sLqvnl2/tYNCWdVUvPI7mbRRTOxYSMOKZmxbOx8Dj3PJvHkZomCqsa+fhAFVdPGxJUPU86ssxNzNd3HOOnr+1h4bhU0uM/60UyeJD7jvVv3s0nv6KBino70RGdt8AnZ8YTZhO2Fp/gsknB93VKqf7qeFMrD7y4neyUGH5z/bQzFg/uK1uI8Nq98zlWe5LLlq3l0sfW0uopp96Qa+0pY8/GMgm87mQb4L6j3FFVQwsjk6PJGhzF3zyt8PmjOx9pFRlmY+KQeLYd0Tq4Uv5ijOHBl3ZS19zGyjtnE91FX21vGJIQxfI7ZvH6jmOkxUeyeHomQxKifHa8QLNMAm9P3M2tpyfwygY780Yl8+svT2H8f76Dw2W6vIkJkDMsgRc2FdPmdBFm8bmAlbKCv20q5oN9FfzHFROYOOTcpn/tjdwRieSOSPT5cfoDy2SwhpYzW+Aul6GqwU5qXAShthCGJQ0C6LIGDjBz+GBa2lzsK9N5UZTytcr6Fv7nzX0sGJNs6bUn+ysLJXBPC9zuPLXtRHMrDpchNdZ9MyQ72T2ZTVe9UKDDjcwjeiNTKV/73/fyaXW6+Pk1k4P2RmIgWS6BN3UooVR6ugO2D43PTnHP+x1m6/oXZUhCFOlxkWzR/uBK+dTJVicvbSnh1tnDTs3Jr7yr2wQuIpEisklEdojIHhH5mWf7SBHZKCIFIvKiiJzbBAbnqL2E0tz6WQv8VAKPc7fA239Jjje1nnVfOcMTtAWulI+V1Z3EZWBqVkKgQwlaPWmB24GLjDHTgOnA5SIyF/g18JgxZjRwArjLd2F+Vvtu6lADrzrVAm8vobgTeMmJk2fdV86wwZTWnqSyvuWs71NK9V5FvfvfZ8duv8q7uk3gxq3R8zTM82OAi4CXPNtXAot9EqHHqRKKvWML3J2A20soPW2Bz9ABPUr5XIWngaQLiftOj2rgImITke1AJfA+cAioNca0N4dLgMwuPrtURPJEJK+qqqrXgTa2nNmNsLLeTmxEKFHh7puWKbER3DFvBE/cPvOs+5qcGUe4LUTnRVHKh8o9CVxb4L7TowRujHEaY6YDWcBsYHxPD2CMedIYk2uMyU1J6d00ri6XobH1zJuYVQ12UuI+G44rIjx89aRTPU26EhFqY3JmnNbBlfKh8roWYiJCu1xkQfXdOfVCMcbUAmuA84AEEWm/MllAqZdjO6Wx1UH7SmgduxHWnmw9NZT+XOUMG8zO0jpaPUuwKaW8q7Kh5VQHA+UbPemFkiIiCZ7HUcClwD7cifx6z9uWAK/6Ksj28gmcPpCnscXR62G5OcMH0+pwseeYzg+ulC+U17WQrvVvn+pJCzwDWCMiO4HNwPvGmDeAh4DvikgBkAQ87asgGzok8I7dCBvtDmJ7m8BP3cjUOrhSvlBRb9cE7mPdZj9jzE5gRifbC3HXw32uvQ94iJxeA2+yO7ucebA76fGRDImPZGvxCe5Ch/gq5U0ul6GivoU0vYHpU5YYidngKZukxEacVgNvsve+hAIwY/hgtumNTKW87rhnmgttgfuWNRK4p4SSFhd5qgZujLtnSl/ucOcMG8yxupYzlmFSSvVNUXUToH3Afc0SCbyxQwJv7wfe3OrEGPqYwN1DfHVAj1Le9fK2UiJCQzhvVFKgQwlqlkjg7TXw9LhImjw3MduH1PelhDJpSDzhoSHaH1wpL2qyO3h1WylXTh1CfFRYoMMJahZJ4A5CxF0Db3W4aHO6TpVS+tICDw8NYWpmPFu0Ba6U16zeWkJTq5Nb53hnxXnVNUsk8Ea7u9bd3tputjtPzYnS1+WZcoYPZk9pPXaHs/s3K6XOqqXNyeNrCpg9IrHbEdGq7yyRwM8blcTX5o8k2jPnSVOrgwa7u6zS12G6OcMSaHW62F2qK/Qo1VerNhVTUW/nu18ci4gu4OBrlkjgl01K54FLxn7WAm91nGqB9z2Bu1sJ27SMolSfvbKtlCmZ8czN1puX/mCJBN6ufdBOk93Z4SZm7wbytEuNiyQzIUp7oijVRyUnmtlRUscVUzMCHcqAYakEPijc3dputDu8chOzXc7wwWw5cgLTPmOWUuqcvbO7HIAvTU4PcCQDh6USeGK0e+bBE82tnyXwSC8k8GEJVNTbOaYDepTqFYfTxYubjzJpSBzDk3T9S3+xVAJP8iTwmsZWmuzuroVRYX0roQDMHK4r1SvVF//YUsLBykbuu2h0oEMZUCyVwBMGhRMiUNNop9HuIDo81Ct3uidkxBEZFsIWTeBKnbMmu4NH3ztA7vDBXDZJyyf+ZKmlMmwhwuBB4VQ3teJwuvrcB7xdmC2EWSMSWXuw90u+KTVQ/XltIdWNdp766kztOuhnlmqBAyTFhFPTaKfJ7vRK/bvdwnGpFFY1cfR4s9f2qVSwO1jRwFNrC7lyasapxcKV/1gvgUdHUNPYSkMfp5L9vIXj3Ot1fpRf6bV9KhXMaptbufvZPKIjQvmPKyYGOpwByXoJPCacmib3TcyYPvYB7yg7OZqhiVF8lK9lFKW643C6uO+FbZTVtvDn22fqyvMBYrkEnhwTQXWj3b2YQ7j3WuAiwsKxqWw4VENLm86LolRXGlra+ObzW/nXwWr++9rJp3pxKf+zXAJPig6nocVBdaPdqzVwcJdRTrY52Vx03Kv7VSpYFFY1svjx9Xy4v5KHr5rIjbk642AgWS+Bx0QAUN3YyoT0OK/u+7xRSYTbQrSMolQnCiobuOGJT6htbuP5u+dwx/m6lmygdZvARWSoiKwRkb0iskdE7vdsf1hESkVku+dnke/DddfA23l7wpxB4aHMyU5kjd7IVOo0eUXHufnJjYgIL31znk5W1U/0pAXuAL5njJkIzAW+LSLtt5wfM8ZM9/y85bMoO0j2JPDYiFAmDvFuCxzgi5PSKaxqYs+xOq/vWymrMcawYv1hbn7yU2IibKxaOoeRyTpUvr/oNoEbY8qMMVs9jxuAfUCmrwPrSlK0u4Qye2QithDvDxq4amoG4bYQXtpS4vV9K2Ulza0OvvPidh5+fS8Lx6Xw6r3zGZ0aG+iwVAfnVAMXkRHADGCjZ9O9IrJTRJ4RkU5vRYvIUhHJE5G8qqq+15ZT4yKIDrdx4fjUPu+rMwmDwrl4QiqvbT9Gm9Plk2Mo1d+tya/ki4+t5dUdx/jepWN58vZcXd+yH+pxAheRGGA18IAxph74EzAKmA6UAY929jljzJPGmFxjTG5KSkqfAx4UHsq6hy7i1tnD+ryvrlyXk0VNUyvrCqp9dgyl+qOGljbuX7WNO5dvJiI0hBeXnsd9F48hxAffdlXf9agfnoiE4U7ezxtjXgYwxlR0eP0p4A2fRNiJwdHh3b+pDy4Ym0xsZChv7CjjwnG+aekr1Z8YY3h3Tzm/eGsfx2pbeOCSMXxz4SgiQr03WE55X7cJXNyz0zwN7DPG/LbD9gxjTJnn6bXAbt+E6H8RoTa+ODGd9/aWY3dM1l9iFdRqGu381xt7+X/bjzE2LYZVS+cya0RioMNSPdCTFvj5wO3ALhHZ7tn2I+AWEZkOGKAI+LpPIgyQK6dlsHprCWsPVHPpxLRAh6OU1zXZHfz+w4OsWF9Em9PFdy8dy7cWjiLUZrnhIQNWtwncGLMO6KwA5pdug4Eyf3QyCYPCeGPnMU3gKqgYY3hjZxm/eHMf5fUtXDcjk29dOEp7mFiQpeYD96cwWwiXT0rn9R3HaGlzEumFlX+UCrRjtSd58KWdrCuoZtKQOB7/So7OZWJh+l3pLK6cOoSmVidr9uvITGV9b+0q4/Jla9lWfIKfXzOJ1+6dr8nb4rQFfhZzsxNJi4vgxbyjfGlKRqDDUapXmuwOfv76Xl7MO8q0rHh+d/MMRuhoyqCgCfwsQm0h3DRrGH/48CBHjzczNHFQoENS6pz8Pe8oj7yzn5qmVr594SgeuGQsYXqTMmjolezGLbOHIsALm4oDHYpS5+TJtYd48KWdZCfHsPqb8/j+ZeM1eQcZvZrdyIiP4uIJafw97yitDh1ar6zh8TUF/M9b+7liagbP3zOHHF2vMihpAu+B2+YOp7qxlXf3lAc6FKW69er2Un7zbj7XTB/C726arq3uIKZXtgcWjE5maGIUz31yJNChKHVWR2qa+MHqXcwekcijN0zTQTlBTq9uD4SECEvOG8GmouNsKz4R6HCU6tITHx/CaQy/v2WGJu8BQK9wD90yexjxUWE88fGhQIeiVKeqGuys3lrKl3OydJX4AUITeA9FR4SyZN4I3t1TQUFlQ6DDUeoMr2wrodXh4u4FulblQKEJ/BzcMW8EkWEh/PnjwkCHotQZNh0+QXZyNKNSYgIdivITTeDnIDE6nJtnDeOVbaUcPd4c6HCUOsUYw7biE8zQ7oIDiibwc/T1L2QTZgvhF2/uC3QoKohUNdjZWnyCuua2Xn3+SE0zNU2t5AxP8HJkqj/TofTnKCM+insvGs1v3s1n3cFq5o9JDnRIysKqG+08ve4wT64txOkyRIfbuO284dw9P5uU2AgcThf1LQ4OVDRwss1JbXMrEaE2Fn1ubp6tnt5ROjnVwKIJvBfumj+Sv+cd5eHX9/D2/Qt0oIQ6J8YYmlqd/OHDg/zlX4dxugzXz8zi0olpvLmzjKfWFrJifRGzRiSy8XANbU5zxj62/+RSEga5lxZ0ugyv7ThGTEQoY3RO7wFFE3gvRIbZ+M8rJnL3s3ms3FDE3QuyAx2SsoD2dScfeSefwuomAG7MzeLuBdmMTXMn3ssmpfPAJWP400eH+PRwDbfOHsbwpGiyU6KJiwrjYEUDD63exbbiWi4cn4oxhu+/tIOP8qv48aIJ2HTx4QFFE3gvXTwhlYXjUlj2wUGunj6E1Fjtd6s+Y4zhsfcPsKOkjpnDB5Nf0cDhqib2ltUzJjWGhy4fz+TMOBaMSTnjs9kpMfzmhmmd7nd8eiw/emU3W46c4MLxqfzunwd5eWsp37lkLPdcoA2JgUYTeC+JCD+5ciKXLVvLr9/O59EbO/8HpwYel8vwyLv5PPHxIeKjwvj4QBVDE6NIiYngl9dN4YaZWb0eJTkoPJQJGbFsLT7Bq9tLWfbBQb6ck8W/XTzay2ehrEATeB9kp8Twtfkj+fPHhdw6Z5jeQFIYY7hv1Tbe3FnGrXOG8fOrJ9FkdxI/KMxrx8gZNphVm46SV3SCOSMT+eV1UxDR0slApHff+ui+i8aQGhvBT17dTZtTp5sd6PYcq+fNnWV8a+EofrF4MqG2EK8mb4B5o5Jpdbq4YGwKf759JuGh+s94oOr2yovIUBFZIyJ7RWSPiNzv2Z4oIu+LyEHPfwdk8zMmIpSHr57EnmP1/N8anSdloHtzVxm2EOHuBdk+axVfNimNj7+/kKe+OvNUTxQ1MPXkT7cD+J4xZiIwF/i2iEwEfgD80xgzBvin5/mAtGhKBounD+EPHx5kf3l9oMNRAWKM4a1dZcwblURitO8Sq4gwPClayyaq+wRujCkzxmz1PG4A9gGZwDXASs/bVgKLfRWkFfz0qknERYXx41d243Kd2W9XBb/95Q0cqWk+Y5CNUr5yTsUzERkBzAA2AmnGmDLPS+VAWhefWSoieSKSV1VV1YdQ+7fB0eH8aNEEthw5wYt5RwMdjgqA9QXVAHxh7JldA5XyhR4ncBGJAVYDDxhjTqsTGGMM0Gmz0xjzpDEm1xiTm5IS3L/YX87JZG52Ir96ez9VDfZAh6P8bMOhGkYmRzMkISrQoagBokcJXETCcCfv540xL3s2V4hIhuf1DKDSNyFah4jw34uncLLNyX0vbNVeKQNIm9PFxsIa5o1KCnQoagDpSS8UAZ4G9hljftvhpdeAJZ7HS4BXvR+e9YxOjeFX103h08Lj/PS1Pbi/nKhgt7OklqZWJ/NG6eRmyn96MpDnfOB2YJeIbPds+xHwK+DvInIXcAS40TchWs91OVkcqGjkiY8PMT49lq+eNyLQISkfcroMv3p7P7ERoZw/Wlvgyn+6TeDGmHVAV/2VLvZuOMHj+5eNo6CygZ+9vpfs5BiddjaIPftJEZuLTvDoDdO0X7byKx3C5SO2EGHZzTMYnRLDt57fwmHP7HP+ll/ewOai41qP95GWNiePrznEvFFJXJeTGehw1ACjCdyHYiJC+cuSXGwhwl0rN1N3snerrfTF0ufyuOGJT1j4m4/Y4OnmprznhU3FVDfa+beLx+jAGuV3msB9bO8Zv04AAA2CSURBVGjiIP5020yKa5q574VtOPzYEj7Z6uRITTOXTUojPDSEW/+ykYdf28PJVqffYghmO0tqeeSdfM7LTmJutta+lf9pAveDudlJ/Nfiyaw9UMV9L2yj1eGfJN5etrlq2hDe+rcF3DFvBCs2FLHo9/9iy5HjfokhWB093szXVuSRFBPO72+ZEehw1AClCdxPbpk9jP+4YgJv7y7nnmfz/NIKPlTVCMColBiiwm08fPUk/nbPHFodLm544hN+9fZ+7A5tjZ+rnSW1LHlmE60OJyvunEVKbESgQ1IDlCZwP7p7QTa/vG4Kaw9WsWT5JhpafFsTL6xqQgRGJkef2jZvVDLvPLCAm2YN5YmPD3HVH9axs6TWp3EEC2MMj68p4JrH19Ngd/CXJbMYrWtQqgDSBO5nt8wexrKbprPlyAlueepTyutafHaswupGhsRHERlmO217bGQYv7xuKsvvnEVtcxtX/3E933huC3uP6UyKXckrOs5Vf1zHb97N56qpQ/jn977A7JGJgQ5LDXCawAPgmumZPPXVmRyuauLqP65jx1F3C3jN/koOVDR47TiFVU1kp0R3+fqF41J5/7tf4P6Lx7D+UDWLfv8vvv5cHnuO1XktBqtzuQx//fQItzz1KSea2nj0hmn87ubpxEV6d5EGpXpD/DnUOzc31+Tl5fnteP3d/vJ67l6ZR1WDnWtnZLJqs3sWw9GpMUwaEsc104dw0fhOJ3k8TXFNM4XVjYgI8VFhxEaGUtvcxu1Pb+TG3KE8fPWkbvdRd7KN5esP8/S6wzS0OLh4fCpLL8hm9sjEAds9bmvxCX722h52lNQxf3Qyj9+a4/XVdZTqCRHZYozJPWO7JvDAqmm0882/bmVT0XEWjElm3qhkthafYMuRExxvauX6mVncsyCbsWkxZyRSp8vwk1d388KmYjqbgjw2MpQVd85i5vCef9WvO9nGyg1FrNhQxPGmVqZlxbP0glFcPjkdW8jASOR1J9v4+et7Wb21hNTYCH64aDyLp2cO2D9kKvA0gfdjrQ4Xb+8u4+IJacREuGc3cDhdLPvgII9/VIAxcMmENH5y5USGJQ069bmn1x3mv97Yy1fPG87V04YA7uTTaHcAsHBcKvFRvWsxtrQ5eWlLCX/5VyFFNc0MSxzE3QtG8uWcLKIjgnct7OKaZpY+l0dBZSP3XJDNty8cfeqaKBUomsAtqqK+hZe2lPCHDw/S6nBx7YwsfvCl8ewureNbz2/lvFFJPL0k12etQ6fL8P7ecv68tpBtxbXERoRy65xhLL0gm6SY4Ok+1+pw8YcPD/LnjwsJDw3hidtm6vw1qt/QBG5x5XUtPOOpUTs99ZLslGj+etccvy0gsOXICVZsKOKNnccIs4Vw2aR0bp41lPOykwixcHmlye7gjuWb2Fx0gmtnZPLQ5eNJj48MdFhKnaIJPEjsOVbHvw5WkxQdzjXTMwkP9X9HooLKBv76aTEvby2hvsXB0MQobsodyvUzh1ou8bW0Oblz+WY2FR3ntzdO45rpOiGV6n80gSuva2lz8u6eclZtOsonhTWEiLvuftOsoVw0PpUwW//upep0Ge55No81+ZX89sZpXDsjK9AhKdWprhK43p1RvRYZZuOa6ZlcMz2TIzVN/D3vKP/IK+HD/ZUkx0SwePoQxqXHMjUrgTGpMf2uzPJRfiUf7q/kp1dN1OStLEkTuPKK4UnRfP+y8XznkrF8lF/Fi3lHWb6h6FS9PmFQGPNHJ3Pl1AwWjks9Y3RoILy8tZTE6HBumzs80KEo1SuawJVXhdpCuGRiGpdMTMPucFJ64iRbi2vZWFjDh/sreWNnGdHhNi6dmMaDl48P2ArudSfbeH9fBbfMGtrvSz1KdUUTuPKZiFAb2SkxZKfEcP3MLBxOFxsPH+eNnWW8ur2UdQXVLL9jNlOy4v0e22PvH6DV4eK6HC2dKOvSpofym1BbCOePTuaX103htXvPJyLUxp0rNlHomfbWX97cWcaKDUXcNX8k04Ym+PXYSnlTtwlcRJ4RkUoR2d1h28MiUioi2z0/i3wbpgo2o1NjWfm1WbQ5DV98bC0PrNrGvjL/zIb47p5y0uMi+dGiCX45nlK+0pMW+Arg8k62P2aMme75ecu7YamBYHRqLG/dv4Al80bw/t4Krv7jOt7eVebz4+4urWNqVvyAmdtFBa9uE7gxZi2g628pn8hMiOI/r5zI+h9cxJTMeO59wbct8fqWNgqrm5iS6f+6u1Le1pca+L0istNTYhnc1ZtEZKmI5IlIXlVVVR8Op4JZwqBwlt8xm+hwG4+8s99nx9lT6v7jMDkAN06V8rbeJvA/AaOA6UAZ8GhXbzTGPGmMyTXG5KakpPTycGogiB8UxrcvHM2a/Co+OVTjk2PsLnUvVqEtcBUMepXAjTEVxhinMcYFPAXM9m5YaqBaMm8EQ+Ij+dU7+/HFNA87SmrJiI8kOYhmUlQDV68SuIhkdHh6LbC7q/cqdS4iw2x859Kx7Dhay+qtpV7dt93h5OP8KuaP1mliVXDodiCPiLwALASSRaQE+CmwUESmAwYoAr7uwxjVAHNdThb/yCvhhy/vJC0uggVjvFN6W3ewmga7g0VTM7p/s1IW0JNeKLcYYzKMMWHGmCxjzNPGmNuNMVOMMVONMVcbY3zf90sNGLYQ4akluYxOjWXps1vYcsQ7naDe3FlGXGQo54/SFrgKDjoSU/VL8VFhPPu12aTFRXDjnz/lZ6/v6VNN3O5w8v7eCi6blB6QOdSV8gX9TVb9VkpsBKu/OY9FUzJYvr6I0tqTvd6Xlk9UMNIErvq1pJgIvn5BNuBe0q233tyl5RMVfDSBq35vfHosg8JtbO1lAi+va+H9PVo+UcFHf5tVvxdqC2H60AS2FJ97Am+yO1jyzCYMcPeCbO8Hp1QAaQJXljBz+GD2lTXQZHec0+dWbT5KfkUDj38lh3HpsT6KTqnA0ASuLGHeqGScLsPqrSU9/ozTZVi5oYiZwwfzhbE6jYMKPprAlSXMzU7kvOwkln1wkMr6lm7ff6z2JP/x/3ZRfLyZO88f4fsAlQoAXVJNWYKI8OMrJnDt/61n/q/XcOW0DM4flYwIREeEMjc7iehwG69sK+WDfRX8c18lALfOGcblk9IDHL1SviG+mDCoK7m5uSYvL89vx1PBp6CykZUbili9tYTmVuep7SECMRGh1Lc4yEyI4tKJadxzQTaZAVo0WSlvEpEtxpjcM7ZrAldW1GR3UN1oB6Cywc6GghpKTjRzycQ0vjgxDRFdbUcFj64SuJZQlCVFR4QSHeH+9R2eFM2sEYkBjkgp/9ObmEopZVGawJVSyqI0gSullEVpAldKKYvSBK6UUhalCVwppSxKE7hSSlmUJnCllLIov47EFJEq4EgvP54MVHsxnP5MzzX4DJTzBD1XXxhujDljSk2/JvC+EJG8zoaSBiM91+AzUM4T9Fz9SUsoSillUZrAlVLKoqyUwJ8MdAB+pOcafAbKeYKeq99YpgaulFLqdFZqgSullOpAE7hSSlmUJRK4iFwuIvkiUiAiPwh0PN4kIkUisktEtotInmdbooi8LyIHPf8dHOg4e0NEnhGRShHZ3WFbp+cmbr/3XOOdIpITuMjPXRfn+rCIlHqu7XYRWdThtR96zjVfRC4LTNS9IyJDRWSNiOwVkT0icr9ne1Bd27OcZ/+5rsaYfv0D2IBDQDYQDuwAJgY6Li+eXxGQ/LltjwA/8Dz+AfDrQMfZy3O7AMgBdnd3bsAi4G1AgLnAxkDH74VzfRj4907eO9HzexwBjPT8ftsCfQ7ncK4ZQI7ncSxwwHNOQXVtz3Ke/ea6WqEFPhsoMMYUGmNagVXANQGOydeuAVZ6Hq8EFgcwll4zxqwFjn9uc1fndg3wrHH7FEgQkQz/RNp3XZxrV64BVhlj7MaYw0AB7t9zSzDGlBljtnoeNwD7gEyC7Nqe5Ty74vfraoUEngkc7fC8hLP/T7QaA7wnIltEZKlnW5oxpszzuBxIC0xoPtHVuQXrdb7XUzZ4pkMpLGjOVURGADOAjQTxtf3ceUI/ua5WSODBbr4xJgf4EvBtEbmg44vG/d0sKPt6BvO5efwJGAVMB8qARwMbjneJSAywGnjAGFPf8bVguradnGe/ua5WSOClwNAOz7M824KCMabU899K4BXcX7kq2r9iev5bGbgIva6rcwu662yMqTDGOI0xLuApPvs6bflzFZEw3EnteWPMy57NQXdtOzvP/nRdrZDANwNjRGSkiIQDNwOvBTgmrxCRaBGJbX8MfBHYjfv8lnjetgR4NTAR+kRX5/Ya8FVPj4W5QF2Hr+OW9Lk677W4ry24z/VmEYkQkZHAGGCTv+PrLRER4GlgnzHmtx1eCqpr29V59qvrGug7vT28G7wI9x3gQ8CPAx2PF88rG/dd6x3AnvZzA5KAfwIHgQ+AxEDH2svzewH3V8w23PXAu7o6N9w9FB73XONdQG6g4/fCuT7nOZeduP9xZ3R4/48955oPfCnQ8Z/juc7HXR7ZCWz3/CwKtmt7lvPsN9dVh9IrpZRFWaGEopRSqhOawJVSyqI0gSullEVpAldKKYvSBK6UUhalCVwppSxKE7hSSlnU/wcjnjSTXwT+2gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(scores)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "178ufOPzay9F",
        "outputId": "3836158e-b35a-471d-c59f-a2c0460712b8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " dense (Dense)               (None, 24)                120       \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 48)                1200      \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 2)                 98        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,418\n",
            "Trainable params: 1,418\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "agent.model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "5E_2klZ3ay9G"
      },
      "outputs": [],
      "source": [
        "agent.env.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "b0TrCnMbay9H"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "colab": {
      "name": "DQNCartPole.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}